#!C:\Python26

from pluginManage import Plugin
import re
import sys
import os
import traceback
import Queue
import threading
import urlparse
import time

import bs4


import logmanager


class KeenStuffScraper(Plugin):

	#Configuration Variables - Loaded when plugin is selected
	capabilities		=	"wScraper"									# Used for detecting ifany random .py file is a plugin file
	version			=	1.4										# Used for detecting available configuration options
	pluginName		=	"KeenStuff"									# Plugin Name variable
	siteName		=	"KeenSpot and ComicGenesis"							# Value for populating "Art Site" dropbox
	artistListEN		=	0										# Determines if Artist List input box is enabled (Boolean, 0-1)
	overwriteModes		=	["Check Folders", "Check Files", "Overwrite Files"]						# Determines if overwrite mode dropbox is enabled for plugin
	retrievalHandles	=	5										# How may http retrieval handles app wants

	DLFolder		=	"KeenSpot and ComicGenesis Webcomics"

	#V1.1 variables:

	aNameLabeltxt		=	"Disabled"
	aListLabeltxt		=	"Disabled"
	#V1.2 variables:

	cookieCapable = 0

	instructionText		=	\
'''Batch Downloads WebComics


'''

	#log = logmanager.ThreadLog(retrievalHandles)

	#V1.3 variables:
	nameSource		=	["KeenSpot", "ComicGenesis"]							# namesource value passed to plugin
	nameSourceEN		=	1										# Determines if Name Source input box is enabled (Boolean, 0-1)


	saveHTML = False


	def getSite(self, startUrl, getHandle):

		scheme, netloc, path, query, fragment = urlparse.urlsplit(startUrl)

		failedLinks = []

		if os.access(os.path.join(self.DLFolder, netloc), os.W_OK) and self.config.ovwMode == "Check Folders":
			self.log.info("self.DLFolder PyDir/" + netloc + "/ Exists: ")
			self.log.info("Assuming site has already been downloaded, skipping")
			return

		#print startUrl
		#print getHandle

		if not os.access(os.path.join(self.DLFolder, netloc), os.W_OK):
			try:
				os.mkdir(os.path.join(self.DLFolder, netloc))
				self.log.info("Created folder " + netloc)
				self.log.info("Beginning Download")
			except:
				self.log.info("Cannot Make working directory %s/. Do you have write Permissions? %s" % (netloc, sys.exc_info()[0]))
				return 0


		retreivedPages = set()
		newPages = set([startUrl])

		retreivedImages = set()
		newImages = set()

		while newPages:
			self.config.progBars.pulseSubBar()
			curPage = newPages.pop()


			mpgctnt, handle = getHandle.getpage(curPage, returnMultiple = True)



			if mpgctnt == "Failed":
				self.log.error("cannot get page")
				failedLinks.append(curPage)
				continue				# skip to the next new page


			retreivedPages.add(curPage)

			if not handle.info()["Content-Type"] == "text/html":
				self.log.error("Incorrect Content-Disposition type!")
				self.log.error("URL - %s" % curPage)

				fname = curPage.rsplit("/")[-1]
				fname = fname.rsplit("?")[0] 		# Sometimes there is some PHP stuff tacked on the end of the Image URL. Split on the indicator("?"), and throw away everything after it.

				self.log.error("Attempting to save file as %s" % fname)

				try:
					filePath = os.path.join(self.DLFolder, netloc, fname)
					fp = open(filePath, "w+b")								# Open file for saving image (Binary)
					fp.write(mpgctnt)
					fp.close()
				except:
					self.log.critical("cannot save image")
					traceback.print_exc()
					print "cannot save file"
				continue



			soup = bs4.BeautifulSoup(mpgctnt, "lxml")

			links = soup.find_all("a", href=True)
			for link in links:
				tmp = link["href"]
				tmp = tmp.split("?")[0]
				if not "http://" in tmp:
					tmp = urlparse.urljoin(startUrl, tmp)
				if netloc in tmp:
					if not tmp in retreivedPages:
						newPages.add(tmp)

			images = soup.find_all("img", src=True)

			newImageNum = 0
			for image in images:
				imTmp = image["src"]

				if not "http://" in imTmp:
					imTmp = urlparse.urljoin(startUrl, imTmp)
				if netloc in imTmp:
					if not (imTmp in newImages | retreivedImages):
						newImageNum += 1
						newImages.add(imTmp)
			self.log.info("Page contains %d images. %d new image(s)." % (len(images), newImageNum))

			self.log.info("Retreived %d images, %d unretreived images. Retreived %d pages, %d pages in queue" % (len(retreivedImages), len(newImages), len(retreivedPages), len(newPages)))

			while newImages:

				self.config.progBars.pulseSubBar()
				imgurl = newImages.pop()

				imScheme, imNetloc, imPath, imQuery, imFragment = urlparse.urlsplit(imgurl)

				fname = imPath.rsplit("/")[-1]
				fname = fname.rsplit("?")[0] 		# Sometimes there is some PHP stuff tacked on the end of the Image URL. Split on the indicator("?"), and throw away everything after it.

				self.log.info("			Filename = " + fname)


				if (not os.access(os.path.join(self.DLFolder, netloc, fname), os.W_OK)) or self.config.ovwMode == "Overwrite Files" :

					self.log.info("Waiting...")
					time.sleep(0.5)


					imgdath = getHandle.getpage(imgurl)							# Request Image





					if imgdath == "Failed":
						self.log.info("cannot get image")

						failedLinks.append(imgurl)
						continue
					self.log.info("Successfully got: " + fname)
					# print fname
					try:
						imagePath = os.path.join(self.DLFolder, netloc, fname)
						fp = open(imagePath, "w+b")								# Open file for saving image (Binary)
						fp.write(imgdath)
						fp.close()
					except:
						self.log.critical("cannot save image")
						traceback.print_exc()
						print "cannot save image"
						failedLinks.append(imgurl)

						continue

					self.log.info("Successfully got: " + imgurl)
				else:
					self.log.info("Exists, skipping...")

				retreivedImages.add(imgurl)
			time.sleep(1)
		return failedLinks


	def childThread(self, getHandle):

		while not self.config.stopChildThreads:
			try:
				comicUrl = self.rootPageQueue.get_nowait()

				self.config.progBars.addSubProgressBar()
				# print "Starting"
				self.log.info("Getting %s's Gallery" % comicUrl)
				faillinks = self.getSite(comicUrl, getHandle)

				if faillinks:
					# print faillinks
					self.failedQueue.put(faillinks)

					faillogptr = open("Failed Pages.txt", "a")
					faillogptr.write("%s%s%s%s%s%s%s" % ("\n\n\n- - - - - - - - - - - Failed Pages of site ", comicUrl, ". Quantity : ", len(faillinks), " at ", time.ctime(time.time()), " - - - - - - - - - - -\n"))
					for link in faillinks:
						faillogptr.write("\n		" + link)
					faillogptr.close()
				time.sleep(1)
				self.log.info("Done")



				self.config.progBars.removeSubProgressBar()

				self.config.progBars.incMainBar()

			except Queue.Empty:
				self.log.info("Thread", threading.currentThread(), "queue is empty. Exiting")
				break

	def getKeenSpotSites(self, getHandle):
		print "KeenSpot"
		print getHandle
		sites = []

		mpgctnt = getHandle.getpage("http://keenspot.com/")

		if self.saveHTML:
			fp2 = open("tempPgLg.html", "wb")
			fp2.write(mpgctnt)
			fp2.close()
																										# Yeah, it's suposed to be misspelled. The misspelling occured IN THE STANDARD, and now it's stuck
		mpgctnt = mpgctnt.decode("utf-8")
		if mpgctnt == "Failed":
			self.log.info("cannot get page")
			return "Failed"


		soup = bs4.BeautifulSoup(mpgctnt, "lxml")
		linkTable = soup.find("table", style="width:100%;width:100%;border-top:1px solid black;border-bottom:1px solid black")
		links = linkTable("a")
		for link in links:

			sites.append(link["href"])
		print len(links)


		return sites

	def getComicGenesisSites(self, getHandle):
		print "ComicGenesis"
		print getHandle

		comicLinks = []


		links =	["Keenspace_0.html", "Keenspace_A.html", "Keenspace_B.html", "Keenspace_C.html",
			"Keenspace_D.html", "Keenspace_E.html", "Keenspace_F.html", "Keenspace_G.html",
			"Keenspace_H.html", "Keenspace_I.html", "Keenspace_J.html", "Keenspace_K.html",
			"Keenspace_L.html", "Keenspace_M.html", "Keenspace_N.html", "Keenspace_O.html",
			"Keenspace_P.html", "Keenspace_Q.html", "Keenspace_R.html", "Keenspace_S.html",
			"Keenspace_T.html", "Keenspace_U.html", "Keenspace_V.html", "Keenspace_W.html",
			"Keenspace_X.html", "Keenspace_Y.html", "Keenspace_Z.html"]

		linkRoot = 'http://guide.comicgenesis.com/'

		for link in links:
			url = urlparse.urljoin(linkRoot, link)
			#print url
			mpgctnt = getHandle.getpage(url)
			if mpgctnt == "Failed":
				self.log.info("cannot get page")
				return "Failed"
			soup = bs4.BeautifulSoup(mpgctnt, "lxml")

			comicLinksRaw = soup("div", class_="comictitle")
			for link in comicLinksRaw:
				comicLink = link.find("strong").find("a")["href"]
				comicLinks.append(comicLink)

			print "Comics = ", len(comicLinksRaw),

			print "Errors = ", len(soup.find_all(text="Error in output"))


			#print comicLinksRaw
		#print len(comicLinks)
		#print comicLinks
		return comicLinks


	def main(self, config, getHandles):											 # Main Function Call

		self.log = logmanager.ThreadLog(self.retrievalHandles)

		self.log.info("Starting")


		self.config = config

		# Check that the path where we're going to save our files is available

		self.DLFolder = os.path.join(config.downloadDir, self.DLFolder)

		if not os.access(self.DLFolder, os.W_OK):									# Creates General Directory
			try:
				os.mkdir(self.DLFolder)
			except:
				self.log.critical("Need write Permissions")
				self.log.critical("%s" % self.DLFolder)

				return


		if self.config.nSrcMode == "KeenSpot":									# Downloads Set of Gallery
			self.log.info("Getting ComicGenesis page list")
			rootPages = self.getKeenSpotSites(getHandles[0])

		elif self.config.nSrcMode == "ComicGenesis":										# Downloads ALL Gallery
			self.log.info("Getting KeenSpot page list")
			rootPages = self.getComicGenesisSites(getHandles[0])



		config.progBars.setupMainBar(len(rootPages))


		self.rootPageQueue = Queue.Queue()
		self.failedQueue = Queue.Queue()

		print "Retreiving", len(rootPages), "sites"

		for name in rootPages:
			self.rootPageQueue.put(name)

		threadNumber = 0
		threads = []

		print "Starting Threads"
		config.stopChildThreads = False

		for getHandle in getHandles:
			temp = threading.Thread(target=self.childThread, name="Thread %d" % threadNumber, args=(getHandle, ))
			threadNumber += 1
			temp.daemon = True
			temp.start()
			threads.append(temp)

		alive = True

		while (not self.rootPageQueue.empty()) or alive:
			alive = False
			for childThread in threads:
				alive = alive or childThread.isAlive()		# Do not exit untill all child threads have returned

			time.sleep(0.1)

		totalfaillinks = []
		while not self.failedQueue.empty():
			totalfaillinks.append(self.failedQueue.get())

		if len(totalfaillinks) > 0:
			self.log.error("- - - - - - - - - - -Overall Failed Pages: %s - - - - - - - - - - -" % (len(totalfaillinks)))
			self.log.error("Done!")
			self.log.error("")


		for link in totalfaillinks:
			self.log.error(link)

		print threads

		logmanager.clearThreadLog(self.log)

		print "exiting"

		return

